# -*- coding: utf-8 -*-
"""
Created on Thu May 17 17:14:41 2018
@author: Alvin Gan
"""

'''
目的：爬腾讯新闻"要闻" 和 "娱乐" tab 下的新闻标题，只爬取首页
生成本地excel文件，并将文件发送的邮箱

'''

print('programe starts to work')

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import datetime as dt
import pandas as pd

import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.header import Header


headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}


class Crawlnewsname(object):   # 爬取腾讯新闻首页
    # url = 'http://news.qq.com/'
    # headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}
    def __init__(self,url):   # 默认返回解析后网站结构
        try:
            self.url = url
            self.res = requests.get(self.url, headers = headers)
            self.res.encoding = 'gbk'
            self.soup = BeautifulSoup(self.res.text, 'html.parser')
        except Exception as e:
            print(e)
        
        
    def major_news(self):   # 解析热门标签新闻
        data = self.soup.select('div[class="item major"]')
        result = []
        for name in data[0].select('a[class="linkto"]'):
            if len(name.text) >0:
                newsdict = {}
                newsdict['name'] = name.text.strip()
                newsdict['link'] = name['href']
                result.append(newsdict)
            else:
                pass
        return result
        
        
    def ent_news(self):    # 解析娱乐标签新闻
        data = self.soup.select('div[class="item ent"]')
        result = []
        for name in data[0].select('a[class="linkto"]'):
            if len(name.text) >0:
                newsdict = {}
                newsdict['name'] = name.text.strip()
                newsdict['link'] = name['href']
                result.append(newsdict)
            else:
                pass
        return result
    

class Sendmail(object):
    def __init__(self, sender, recevers, server, port = 465):
        self.sender = sender
        self.recevers = recevers
        self.server = server
        self.port = port
    

    def sendmail(self, title, text, path, name):
        message = MIMEMultipart()  # 创建一个可发送附件的对象
        message['From'] = self.sender
        message['To'] = ';'.join(self.recevers) # 收件人需要是字符串，用分号将收件人地址隔开
        message['Subject'] = Header(title, 'utf-8')  # 邮件主题
        message.attach(MIMEText(text, 'plain','utf-8'))  # 邮件正文
        # 打开要发送的附件，注意格式编码
        att = MIMEText(open(path + '\\' + name, 'rb').read(), 'base64', 'utf-8')
        #这两行不要写错了
        att['Content-Type'] = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
        att['Content-Disposition'] = "attachment; filename =%s" % name
        # 将附件添加到message
        message.attach(att)
        # 尝试发送邮件
        try:
            server = smtplib.SMTP_SSL(self.server, self.port)
            server.login(self.sender, '*******') # 星号是你的邮箱的授权码，填写自己的
            server.sendmail(self.sender, self.recevers, message.as_string())
            server.quit()
            print("邮件发送成功，请注意查收!")
        except Exception as e:
            print('邮件发送失败，错误消息如下:')
            print(e)
        return True



def to_excel(df, path, result_time):   # 输入dateframe到excel
    df['date'] = result_time  # 添加日期字段
    df['name'] = '[阅读+0.01]' + df['name']
    names = ['date','catagory','name','link']
    df = df.reindex(columns = names)
    df.to_excel(path + '\\' + result_time + '.xlsx', index = False)   # 以日期命名文件
    
print('working hard...')

if __name__ == '__main__':
    # 得到字符串的日期  yyyy-mm-dd
    ctime = datetime.now()
    delta = dt.timedelta(days = 3)  # 日期顺延三日
    next_time = ctime + delta
    result_time = next_time.strftime('%Y-%m-%d') # 日期转为字符串 yyyy-mm-dd格式
    
    print('正在爬取新闻标题...')
    # 爬取新闻标题
    url = 'http://news.qq.com/'
    news = Crawlnewsname(url)  # 创建对象
    major = news.major_news()  # 爬要闻
    ent = news.ent_news()   # 爬娱乐新闻
    
    print('正在整理爬取的数据...')
    # 使用pandas整理得到的数据
    major_df = pd.DataFrame(major)    # 使用pandas整理数据
    major_df['catagory'] = 'major' # 添加固定字段
    ent_df = pd.DataFrame(ent)
    ent_df['catagory'] = 'ent'
    my_df = pd.DataFrame({'catagory': None, 'name':'您昨天的收益已到账，请点击查看', 'link': None}, index = [0])
    combine_df = pd.concat([my_df,major_df, ent_df])  # 合并热门新闻和娱乐新闻
    
    print('正在将爬取的数据输出到本地...')
    # 将dataframe输出为excel到本地
    path = r'D:\日常数据\日常push'
    to_excel(combine_df, path, result_time)
    
    print('正在发送邮件')
    # 发送邮件
    sender = '***@**.com'
    receivers = ['***@**.com', '***@**.com']
    server = 'smtp.exmail.qq.com'
    title = '每日推送新闻标题%s' % result_time
    text = '你好，附件为%s日的新闻标题，请查收' % result_time
    
    mail = Sendmail(sender, receivers, server)
    mail.sendmail(title, text, path, result_time + '.xlsx')
    
print('working is done!')